# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy
import re
from scrapy.loader.processors import MapCompose, TakeFirst, Identity, Join
from scrapy.loader import ItemLoader


class ArticlespiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


def add_cnblog(value):
    return value + "-connor"


def add_cnblog2(value):
    return value + "+connor"


# 自定义日期类型解析
def date_convert(value):
    match_re = re.match(".*?(\d+.*)", value)
    if match_re:
        return match_re.group(1)

    return "1970-07-01"


# 自定义去除[]中某个值
# def custom_remove_item(value):
#     if value == "linux":
#         return ""
#     else:
#         return value


class ArticleItemLoader(ItemLoader):
    """
    自定义ItemLoader解析器
    """
    default_output_processor = TakeFirst()


# 定义items在yield中回调方法直接传递
class CnBlogsArticleItem(scrapy.Item):
    title = scrapy.Field(
        # 在插入数据之前回调
        # input_processor=MapCompose(add_cnblog, add_cnblog2),
        # 获取非空、非null到值
        # ouput_processor=TakeFirst()
    )
    create_date = scrapy.Field(
        input_processor=MapCompose(date_convert)
    )

    url = scrapy.Field()
    url_object_id = scrapy.Field()
    front_image_url = scrapy.Field(
        # url已经配置[]类型，不能再使用 TakeFirst获取第一个。
        # 这里设置使用默认 Identity()即不做处理
        output_processor=Identity()
    )
    front_image_path = scrapy.Field()
    praise_num = scrapy.Field()
    comment_num = scrapy.Field()
    fav_num = scrapy.Field()
    tags = scrapy.Field(
        # 自动将[]join返回字符串,逗号分割
        output_processor=Join(separator=',')
    )
    content = scrapy.Field()


class ZhihuQuestionItem(scrapy.Item):
    """
    知乎问答item
    """
    zihhu_id = scrapy.Field()
    topics = scrapy.Field()
    url = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()
    # create_time = scrapy.Field()
    # update_time = scrapy.Field()
    answer_num = scrapy.Field()
    comments_num = scrapy.Field()
    watch_user_num = scrapy.Field()
    click_num = scrapy.Field()
    crawl_time = scrapy.Field()
    # crawl_update_time = scrapy.Field()


class ZhihuAnswerItem(scrapy.Item):
    """
    知乎答案item
    """
    zhihu_id = scrapy.Field()
    url = scrapy.Field()
    question_id = scrapy.Field()
    author_id = scrapy.Field()
    author_name = scrapy.Field()
    content = scrapy.Field()
    praise_num = scrapy.Field()
    comment_num = scrapy.Field()
    # create_time = scrapy.Field()
    # update_time = scrapy.Field()
    crawl_time = scrapy.Field()
    # crawl_update_time = scrapy.Field()
